# -*- coding: utf-8 -*-
'''
Created on 2020-12-17

@author: qqyu
'''
import json
import time
import math
from requests import get,post
import sys
import os
import traceback
import openpyxl
import re
from pathlib import Path
from multiprocessing.dummy import Pool
from hk_utils import is_current_year_month,millo_to_yearmonthday,str_to_millisecond

class GlobalConstants(object):
    access_token,expires_in = '',math.floor(time.time())
    #时间戳工具：https://tool.lu/timestamp/
    def __init__(self,appkey='',appsecret=''):
        if appkey=='':
            # self.appkey,self.appsecret = "dingtawyurmk0j1gixvp","WMoJfu8Tq2S8p7e0-FazC1gSIT5O2m3sB1TGi54elc7hNyV41oN_PveWnx1DAGOT"
            path = Path.cwd().joinpath('HKSalary配置文件.xlsx')
            setting_st = list(openpyxl.load_workbook(path)['Sheet1'].values)[1:]
            self.appkey,self.appsecret,startend_time,self.common_filter,self.kq_sheetname,self.process,self.has_fujian,self.invalid_instance_marker = [i[2] for i in setting_st]
            self.common_filter = self.common_filter.split('、')
            self.start_time,self.end_time = startend_time.split('----')
            self.invalid_instance_marker = self.invalid_instance_marker.split('、')
            self.has_fujian = self.has_fujian == '是' or False
            if self.appkey == None or self.appsecret == None:
                raise Exception('appkey/appsecret不能为空，可以去 https://open-dev.dingtalk.com/#/corph5 网站获取')
            if self.kq_sheetname == None:
                input('没有考勤表，继续将不记录[算薪天数、病假天数、应计薪天数]？(输入回车继续)')
            if self.process==None:
                self.process = 6
            else:
                int(self.process)
        else:
            self.appkey,self.appsecret = appkey,appsecret
        #
        tm_mon = (time.localtime().tm_mon <10 and '0{}'.format(time.localtime().tm_mon)) or time.localtime().tm_mon
        paths = [str(time.localtime().tm_year),str(time.localtime().tm_year)+str(tm_mon),'5-工资汇总表.xlsx']
        self.fname = Path.cwd().joinpath(*paths)
        paths = [str(time.localtime().tm_year),str(time.localtime().tm_year)+str(tm_mon),'1-4-工资汇总表.xlsx']
        self.fname1 = Path.cwd().joinpath(*paths)
        paths = [str(time.localtime().tm_year),str(time.localtime().tm_year)+str(tm_mon),self.kq_sheetname]
        self.fname2 = Path.cwd().joinpath(*paths)

class HKBaseCrawl(GlobalConstants):

    def __init__(self):
        super().__init__()
        #初始化access_token
        self.get_access_token()

    def crawl(self,url,_data,method='post') -> dict:
        access_token = self.get_access_token()
        _url = url + "?access_token={}".format(access_token)

        if method == 'get':
            resp = get(_url,json=_data).json()
        else:
            resp = post(_url,json=_data).json()
        if resp['errmsg'] != 'ok':
            raise Exception(str(resp))
        return resp

    def get_access_token(self) -> str:
        '''
        获取access_token,实时更新
        :return:access_token
        '''
        if self.access_token=='' or int(time.time()) > self.expires_in:
            _url = 'https://oapi.dingtalk.com/gettoken?appkey={appkey}&appsecret={appsecret}'
            resp = get(_url.format(appkey=self.appkey,appsecret=self.appsecret)).json()
            # print(resp)
            if resp['errmsg'] != 'ok':
                raise Exception(str(resp))
            self.access_token,self.expires_in = resp['access_token'],int(time.time())+resp['expires_in']
        
        return self.access_token

class HKEmployeeCrawl(HKBaseCrawl):
    
    def get_queryonjob(self,status_name='所有') -> list:
        '''获取在职员工列表：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/smartwork/hrm/employee/queryonjob
            获取智能人事查询公司离职员工id列表：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/smartwork/hrm/employee/querydimission
            2：试用期
            3：正式
            5：待离职
            -1：无状态
            离职
        '''
        ls = []
        _data = {"offset":'0',"size":'50'}#50最大条数
        while True:
            
            if '离职' in status_name:
                _url = "https://oapi.dingtalk.com/topapi/smartwork/hrm/employee/querydimission"
            else:
                _url = "https://oapi.dingtalk.com/topapi/smartwork/hrm/employee/queryonjob"
                if status_name=='所有':
                    _data['status_list'] = '2,3,5,-1'
                else:
                    status_list={'试用期':'2','正式':'3','待离职':'5','无状态':'-1'}[status_name]
                    _data['status_list'] = status_list
            
            resp = self.crawl(_url,_data)

            ls = ls + resp['result']['data_list']
            if not 'next_cursor' in resp['result']:
                break
            _data['offset'] = resp['result']['next_cursor']

        return ls

    def get_querydimission(self) -> list:
        '''获取智能人事查询公司离职员工id列表：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/smartwork/hrm/employee/querydimission
        '''
        ls = []
        _data = {'offset':'0','size':'50'}#最大50
        while True:
            _url = 'https://oapi.dingtalk.com/topapi/smartwork/hrm/employee/queryonjob'
            
            resp = self.crawl(_url,_data)

            ls = ls + resp['result']['data_list']
            if not 'next_cursor' in resp['result']:
                break
            _data['offset'] = resp['result']['next_cursor']
        return ls

    def get_employeelistdimission(self,userid_list='6605055560775167') -> dict:
        '''批量获取员工离职信息：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/smartwork/hrm/employee/listdimission
        '''
        ls = {}
        while True:
            _url = 'https://oapi.dingtalk.com/topapi/smartwork/hrm/employee/listdimission'
            _data = {'userid_list':userid_list}  

            resp = self.crawl(_url,_data)

            _ls = {i['userid']: is_current_year_month(millo_to_yearmonthday(i['last_work_day'])) + '```' + millo_to_yearmonthday(i['last_work_day']) for i in resp['result']}
            for k in _ls:
                ls[k] = _ls[k]
            break
        return ls

    def employee_list(self,userid_list="213419155321041550,213415331021170549",field_filter_list=['姓名','工号','入职时间','员工状态']) -> dict:
        '''批量获取员工花名册字段信息：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/smartwork/hrm/employee/list
        '''
        ls = {}
        while True:
            _url = 'https://oapi.dingtalk.com/topapi/smartwork/hrm/employee/list'
            _data = {'userid_list':userid_list,'field_filter_list':field_filter_list}#最大50

            resp = self.crawl(_url,_data)

            _ls = [i['field_list']+[{'userid':i['userid']}] for i in resp['result']]
            for i in _ls:#[]27个
                _tmp = '```'.join([j['value'] for j in i if 'field_name' in j and j['field_name'] in field_filter_list and 'value' in j])
                _userid = [j['userid'] for j in i if 'userid' in j][0]
                if _tmp.split('```')[1] == '':
                    continue
                iscurrentmonth = is_current_year_month(_tmp.split('```')[-2])
                _tmp = _tmp + '```' + iscurrentmonth
                print(_tmp)
                ls[_userid] = _tmp
            break
        #"name":"高祥```013```2020-12-19```5```是
        return ls #姓名```工号```入职时间```在职状态(4为离职)```是否当月入职

    def get_employee_ls(self,p_result,userid_list,pool):
        '''根据userid更新在职/离职用户信息，在职员工一次请求最多50个员工'''
        all_ls = list(range(0,len(userid_list.split(',')),50))+[len(userid_list.split(','))]
        all_userid_list = [','.join(userid_list.split(',')[all_ls[i]:all_ls[i+1]]) for i in range(0,len(all_ls)-1)]
        em_detail_dict = pool.map(lambda u_list:self.employee_list(userid_list=u_list),all_userid_list)
        em_detail_dict = {j:i[j] for i in em_detail_dict for j in i}#flatMap
        for i in p_result:
            _userid = i['originator_userid']
            try:
                i['name'] = em_detail_dict[_userid]
            except Exception as err:
                pass# print('此员工信息出现错误 =>',i,err.args)
                
        p_result = [i for i in p_result if 'name' in i]##去除异常审批记录，没有员工信息或者审批信息的审批记录
        # #离职员工用户列表去重
        status4_userid_list = [i['originator_userid'] for i in p_result if 'name' in i and i['name'].split('```')[-2]=='4']
        if status4_userid_list != '':
            ##根据userid查询离职员工信息
            all_ls = list(range(0,len(status4_userid_list),50))+[len(status4_userid_list)]
            all_userid_list = [','.join(status4_userid_list[all_ls[i]:all_ls[i+1]]) for i in range(0,len(all_ls)-1)]
            em_dimidetail_dict = pool.map(lambda u_list:self.get_employeelistdimission(userid_list=u_list),all_userid_list)
            em_dimidetail_dict = {j:i[j] for i in em_dimidetail_dict for j in i}#flatMap
        else:
            em_dimidetail_dict = {}
        
        for i in p_result:
            _userid = i['originator_userid']
            if _userid in em_dimidetail_dict:### 离职员工
                i['name'] = i['name'] + '```' + em_dimidetail_dict[_userid]
            else:
                i['name'] = i['name'] + '```{}```'.format('否')
        p_result = [i for i in p_result if 'name' in i and len(i['name'].split('```')) == 7]##再次去除异常审批记录，没有员工信息或者审批信息的审批记录
        return p_result

class HKFileCrawl(HKBaseCrawl):

    def process_file_url_get(self,process_instance_id="89b61087-fbff-4c60-b7da-30904c0c6922",fileid_ls='27756812418') -> dict:
        '''获取单个审批实例详情：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/processinstance/file/url/get
        '''
        ls = {process_instance_id:[]}
        for file_id in fileid_ls.split(','):
            _data = {
                        "request":{
                            "process_instance_id":process_instance_id,
                            "file_id":file_id
                        }
                    }
            _url = 'https://oapi.dingtalk.com/topapi/processinstance/file/url/get'
            
            resp = self.crawl(_url,_data)

            if resp['errmsg'] != 'ok':
                raise Exception(str(resp))
            if 'download_uri' in resp['result']:
                ls[process_instance_id].append(resp['result']['download_uri'])
        
        return {i:','.join(ls[i]) for i in ls}

    def get_process_file_ls(self,p_result,pool):
        '''根据fileId更新文件下载地址信息'''
        ##根据fileId查询下载地址(文件有效期15分钟)
        process_instance_id_ls = [(i['process_instance_id'],i['附件']) for i in p_result if '附件' in i and i['附件']!='' and '通用审批' in i['title'] and i['金额（元）'] !='' ]
        process_instance_id_ls = pool.map(lambda kv:self.process_file_url_get(process_instance_id=kv[0],fileid_ls=kv[1]),process_instance_id_ls)
        process_instance_id_ls = {j:i[j] for i in process_instance_id_ls for j in i}#flatMap
        for i in p_result:
            # process_instance_id = i['process_instance_id']
            if 'process_instance_id' in i and i['process_instance_id'] in process_instance_id_ls:
                i['附件'] = process_instance_id_ls[i['process_instance_id']]
        return p_result

class ExcelPipeLine(GlobalConstants):

    def mapandsave_process(self,p_result):
        '''清洗所有审批记录，并保存到一个sheet中'''
        wb = openpyxl.Workbook()
        sheet = wb.active
        sheet.title = '审批记录留存表'
        much_rows = []
                        #0	     1	        2	        3	        4	      5	        6	    7	    8	      9	            10	        11	            12	        13	        14	            15	        16	        17	            18          19
        sheet_titles = ['工号', '姓名', '算薪天数', '病假天数', '应计薪天数', '提成', '课时费', '运营奖励', '奖金', '入职时间', '是否当月入职', '是否当月离职', '离职时间', '调整前薪资', '调整后薪资', '调薪生效日期', '转岗前工资', '转岗后工资', '转正是否调薪','审批最终完成时间']
        for p_msg_dict in p_result:
            one_row = [None for i in range(20)]
            # "高祥```013```2020-12-19```5```是```否```-"
            ##姓名```工号```入职时间```在职状态(4为离职)```是否当月入职```是否当月离职```离职时间
            _name,_jobnum,_injobdate,_t,_iscurrentmonthinjob,_iscurrentmonthleavejob,_leavejobdate = p_msg_dict['name'].split('```')#用户信息
            title = ''
            if 'title' in p_msg_dict:
                title = p_msg_dict['title']
            one_row[0] = _jobnum
            one_row[1] = _name
            one_row[9] = _injobdate
            one_row[10] = _iscurrentmonthinjob
            one_row[11] = _iscurrentmonthleavejob
            one_row[12] = _leavejobdate
            if 'finish_time' in p_msg_dict:
                one_row[19] = int(p_msg_dict['finish_time'])
            else:
                one_row[19] = int(str_to_millisecond('2000/10/10'))
            if '提成、课时费、代班费审批' in title:
                _1,_2,_3,_4 = p_msg_dict['金额（元）'].split('```')
                one_row[5] = _1
                one_row[6] = _2
                one_row[7] = _3
                one_row[8] = _4
            elif '晋级/降级审批' in title and '调整前薪资' in p_msg_dict:
                _1,_2,_3 = p_msg_dict['调整前薪资'], p_msg_dict['调整后薪资'], p_msg_dict['生效日期']
                one_row[13] = _1
                one_row[14] = _2
                one_row[15] = _3
            elif '调动审批' in title and '转岗前工资' in p_msg_dict:
                _1,_2 = p_msg_dict['转岗前工资'], p_msg_dict['转岗后工资'] #p_msg_dict['转正是否调薪```']
                one_row[16] = _1
                one_row[17] = _2
                # one_row[18] = _3
            much_rows.append(one_row)
        sheet.append(sheet_titles)
        #修改完成时间为日期类型
        # print(much_rows)
        [sheet.append(row[0:-1]+[millo_to_yearmonthday(row[-1])]) for row in much_rows]
        wb.save(self.fname)
        return much_rows,sheet_titles

    def mapandsave_common_process(self,much_rows,sheet_titles,p_result):
        '''
        保存通用金额表表审批数据
        '''
        wb = openpyxl.load_workbook(self.fname)
        sheet = wb.create_sheet('通用金额表') #通用审批中含有钱的审批
        # sheet.append(sheet_titles)
        ls = [i for i in p_result if '通用审批' in i['title'] and i['金额（元）'] !='']
        if len(ls)>0:
            sheet.append(list(ls[0].keys()))#添加表头
            ls = [list(row.values()) for row in ls]
            [sheet.append(row[0:-3]+[millo_to_yearmonthday(row[-3]),row[-2]]) for row in ls]
        wb.save(self.fname)
    
    # 读取离线1-考勤表.xlsx
    def join_kaoqin_excel(self,excela_ls):
        '''合并审批和考勤表'''
        if self.fname2 == '':
            return excela_ls
        excela_ls = [list(i) for i in excela_ls]
        titles = list(openpyxl.load_workbook(self.fname2)['Sheet1'].values)
        titles_index = [list(titles[0]).index(i) for i in ['工号','算薪天数','病假天数','应计薪天数']]
        excelb_ls = [[i[j] for j in titles_index] for i in titles[1:]]
        excelb_index_ls = [i[0] for i in excelb_ls]
        for row in excela_ls:
            try:
                _index = excelb_index_ls.index(row[0])
                rowb = excelb_ls[_index]
                row[2] = rowb[1]
                row[3] = rowb[2]
                row[4] = rowb[3]
            except Exception:
                print(' 考勤表中没有{}工号'.format(row[0]))
            print('合并考勤成功...')
        return excela_ls
    
    def reduceandsave_process(self,much_rows,sheet_titles):
        '''将所有的审批记录，按员工工号聚集在一起'''
        ###聚合审批记录，   5	     6	      7	       8       19
        ################'提成', '课时费', '运营奖励', '奖金'  '审批最终完成时间'
        merge_much_rows = []
        wb = openpyxl.load_workbook(self.fname)
        sheet = wb.create_sheet('审批汇总表') #通用审批中含有钱的审批

        _users_dict = {row[0]:[] for row in much_rows}
        for row in much_rows:
            _users_dict[row[0]].append(row)
        for _jobnum in _users_dict:
            _userls = _users_dict[_jobnum]
            _userls = sorted(_userls,key = lambda i: i[-1],reverse=True)
            newrow = _userls[0] ##时间为最后的一条审批
            #'提成、课时费、代班费审批'
            calc_sum = lambda num:sum([float(row[num]) for row in _userls if row[num]!=None])
            newrow[5] = calc_sum(5)
            newrow[6] = calc_sum(6)
            newrow[7] = calc_sum(7)
            newrow[8] = calc_sum(8)
            #'晋级/降级审批'13	        14	            15	        
            _13_14_15_ls = sorted([row for row in _userls if row[13]!=None and row[14]!=None],key = lambda i: i[-1],reverse=True)
            if len(_13_14_15_ls) > 0:
                newrow[13] = _13_14_15_ls[0][13]
                newrow[14] = _13_14_15_ls[0][14]
                newrow[15] = _13_14_15_ls[0][15]

            #'调动审批'16	        17
            _16_17_ls = sorted([row for row in _userls if row[16]!=None and row[17]!=None],key = lambda i: i[-1],reverse=True)
            if len(_16_17_ls) > 0:
                newrow[16] = _16_17_ls[0][16] 
                newrow[17] = _16_17_ls[0][17]
            del newrow[-1]
            merge_much_rows.append(newrow)
        sheet.append(sheet_titles[0:-1])
        ##TODO 待加入
        merge_much_rows = self.join_kaoqin_excel(merge_much_rows)
        [sheet.append(row) for row in merge_much_rows]
        wb.save(self.fname)

    ###留存备份###
    def save_archives(self,p_result):
        '''将所有的审批，按照审批类型保存到一个excel表格中'''
        wb = openpyxl.Workbook()
        with open(r'{}.txt'.format('p_result'),'w',encoding='utf-8') as f:
            f.write(json.dumps(p_result,ensure_ascii=False))
        excel_names = set([i['title'].split('提交的')[1] for i in p_result if not i['title'].startswith('-')])
        for name in excel_names:
            _tmp = [i for i in p_result if name in i['title'] and (not i['title'].startswith('-'))]
            if len(_tmp)==0:
                continue
            _tmpsheet = wb.create_sheet(name.replace('/','-'))
            _tmpsheet.append(list(_tmp[0].keys()))
            [_tmpsheet.append(list(row.values())) for row in _tmp]
        wb.save(self.fname1)      

    def insertByBatch(self,p_result):
        '''汇总审批+员工信息+附件文件成一个表格'''
        much_rows,sheet_titles = self.mapandsave_process(p_result)
        self.mapandsave_common_process(much_rows,sheet_titles,p_result)
        self.reduceandsave_process(much_rows,sheet_titles)
        os.system(str(self.fname))
        # self.save_archives(p_result)

class HKProcessCrawl(HKBaseCrawl):

    def __init__(self):
        super().__init__()
        self.hkEmployeeCrawl = HKEmployeeCrawl()
        self.hkFileCrawl = HKFileCrawl()
        self.pool = Pool(processes=self.process)

    def get_listids(self,process_code="PROC-A7B82F4A-6834-4E6D-9633-D7E19638E593",start_time='2020/11/11',end_time='2020/12/21') -> list:
        '''分页获取审批实例id列表：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/processinstance/listids
        根据审批开始时间和结束时间来查询
        '''
        _data = {'process_code':process_code,
                            'start_time':str_to_millisecond(start_time),
                            'end_time':str_to_millisecond(end_time),
                            'cursor':'0','size':'10'}#最多返回10条记录
        ls = []
        while True:
            _url = 'https://oapi.dingtalk.com/topapi/processinstance/listids'
            
            resp = self.crawl(_url,_data)

            ls = ls + resp['result']['list']
            if not 'next_cursor' in resp['result']:
                break
            _data['cursor'] = resp['result']['next_cursor']
        return ls

    def process_instance_get(self,process_instance_id="de5c7410-0e00-4a88-aa86-593d0fc40584",common_filter=['元','工资','奖金','金额']) -> dict:
        '''获取单个审批实例详情：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/processinstance/get
        '''
        ls = []
        _data = {'process_instance_id':process_instance_id}#最大50
        while True:
            _url = 'https://oapi.dingtalk.com/topapi/processinstance/get'   
            resp = self.crawl(_url,_data)

            ls = resp['process_instance']
            break
        if ls['status'] != 'COMPLETED':return {}
        ###TODO 过滤审批通过的但是是重复的审批
        if len(list(filter(lambda i: i in str(ls['operation_records']),self.invalid_instance_marker))) > 0:return {}
        ##去掉了id，有可能失去了唯一性，可能会重复+'```'+i['id']
        _ls = {i['name']:i['value'] for i in ls['form_component_values'] if 'name' in i and 'id' in i}
        _ls['originator_userid'] = ls['originator_userid']
        _ls['process_instance_id'] = process_instance_id
        _ls['title'] = ls['title']
        _ls['finish_time'] = int(str_to_millisecond(ls['finish_time']))#"2020-12-22 21:26:23
        if '通用审批' in ls['title']:
            _ls['金额（元）'] = ''
        
        for i in _ls:
            if i.startswith('图片'):
                if _ls[i]=='null':
                    _ls[i]=''
                else:
                    _ls[i] = ','.join(json.loads(_ls[i]))
            elif i.startswith('附件'):
                if _ls[i]=='null':
                    _ls[i]=''
                else:
                    _tmp1 = json.loads(_ls[i])
                    _ls[i] = ','.join(list(map(lambda x: x['fileId'],_tmp1)))#+'```'+x['fileName']
            #TODO 更多字段的设置和清洗
            elif i.startswith('金额（元）') and '提成、课时费、代班费审批' in str(_ls):
                _tmp = ['0','0','0','0']#提成	课时费	运营奖励	奖金
                _ctx = _ls['申请事项']+_ls['审批详情']
                if '提成' in _ctx:
                    _tmp[0] = _ls['金额（元）']
                elif '课时费' in _ctx:
                    _tmp[1] = _ls['金额（元）']
                elif '运营奖励' in _ctx:
                    _tmp[2] = _ls['金额（元）']
                elif '奖金' in _ctx:
                    _tmp[3] = _ls['金额（元）']
                _ls[i] = '```'.join(_tmp)
            elif i.startswith('title') and '通用审批' in _ls[i]:
                # _tmp = ''#通用金额，提成	课时费	运营奖励	奖金
                _ctx = _ls['申请内容']+'\n'+_ls['审批详情']
                # common_filter=['元','工资','奖金','金额']
                if len([x for x in common_filter if x in _ctx])>0:
                    _ls['金额（元）'] = ','.join(re.findall('(\\d+?|\\d+.\\d+?)元',_ctx,re.S))
        
        return _ls

    def get_listbyuserid(self,userid="213419155321041550",p_tmp_ls=['调动审批','新员工入职审批','晋级/降级审批','通用审批','员工转正审批','提成、课时费、代班费审批','实习生转正审批']) -> dict:
        '''获取指定用户可见的审批表单模板列表：
        接口api地址 https://open-dev.dingtalk.com/apiExplorer#/?devType=org&api=/process/listbyuserid
        p_tmp_ls 审批模板名称
        '''
        ls = []
        _data = {'userid':userid,'offset':'0','size':'100'}#最大50
        while True:
            _url = 'https://oapi.dingtalk.com/topapi/process/listbyuserid'
            
            resp = self.crawl(_url,_data)

            ls = ls + resp['result']['process_list']
            if not 'next_cursor' in resp['result']:
                break
            _data['offset'] = resp['result']['next_cursor']

        return {i['name']:i['process_code'] for i in ls if i['name'] in p_tmp_ls}#TODO 仅仅测试使用 if i['name'] in p_tmp_ls

    def get_process_detail_ls(self,start_time='2020/11/11',end_time='2020/12/28'):
        '''获取审批详情列表'''
        process_tmp_ls = self.get_listbyuserid()#获取审批模板列表{key:value,...}
        print(process_tmp_ls)    
        # #分页获取审批实例id列表
        process_instance_ids_ls = self.pool.map(lambda p_name: self.get_listids(process_code=process_tmp_ls[p_name],start_time=start_time,end_time=end_time),process_tmp_ls)
        process_instance_ids_ls = [j for i in process_instance_ids_ls for j in i]#flatMap操作
        ###上千次请求-大量的阻塞请求 #根据审批示例id获取详情
        p_result = self.pool.map(lambda p_id: self.process_instance_get(process_instance_id=p_id,common_filter = self.common_filter),process_instance_ids_ls)
        p_result = list(filter(lambda i: len(i)>0,p_result)) #去掉未完成的审批
        # #获取在职员工列表，并用户列表去重(在职员工+全部审批=所有应该发工资的员工)
        # # #TODO 如果员工不申请审批，被动被管理员删除。那么需要主动查询所有离职员工的信息进行检索离职时间为当月的
        allonjob_ids = set(self.hkEmployeeCrawl.get_queryonjob())
        onthejob_list = allonjob_ids - set([i['originator_userid'] for i in p_result])
        p_result = p_result + [{'originator_userid':i,'title':'-提交的-'} for i in onthejob_list]
        userid_list = ','.join(allonjob_ids | set([i['originator_userid'] for i in p_result]))
        return p_result,userid_list

    def processJoinEmployeeAndFile(self,p_result, userid_list):
        p_result = self.hkEmployeeCrawl.get_employee_ls(p_result, userid_list,self.pool)
        if self.has_fujian:
            p_result = self.hkFileCrawl.get_process_file_ls(p_result,self.pool)                 #根据fileId更新文件下载地址信息
        
        return p_result

def main():
    print('程序开始...')
    sys.excepthook = lambda exc_type, exc_value, exc_obj: input('运行发生异常===》'+'{}'.format(traceback.print_exception(exc_type,exc_value,exc_obj,limit=20,file=sys.stdout)))
    mdt = HKProcessCrawl()
    excelPipeLine = ExcelPipeLine()
    starttime = time.time()
    
    p_result,userid_list = mdt.get_process_detail_ls(start_time=mdt.start_time,end_time=mdt.end_time)#获取审批详情列表
    p_result = mdt.processJoinEmployeeAndFile(p_result,userid_list)
    
    excelPipeLine.insertByBatch(p_result)                       #汇总审批+员工信息+附件文件成一个表
    print('耗时：',time.time()-starttime)
    input('按回车退出！')

if __name__ == "__main__":
    main()